<?php

include_once('/var/www/vhosts/designexpress.be/httpdocs/Classes/File/Html/SimpleHtmlDom.php');
require_once "/opt/www/designexpress/web/www.designexpress.be/Classes/DB/Pdo.php";
require_once "/opt/www/designexpress/web/www.designexpress.be/Classes/Output/Autokeyword.php";

Class PHP_Curl_Scraper{
	protected $morepagestoparse;
	protected $site;
	protected $pagecounter; // keeps amount of pages parsed

        /**
         * PHP_Curl_Scraper verwacht een url als input
         * maakt tabel crawler en crawler_images leeg.
         * @param String $site
         */
	function __construct($site){
		$this->site = $site;
		// connect to database
                $this->prepareTables();
		$this->morepagestoparse = 0;
		$this->pagecounter = 0;
		$this->urlsfromlastpage = array();
	}

        function prepareTables(){
                try{
                $pdo = new DB_Pdo('vandersanden');
		$result=$pdo->prepare("TRUNCATE TABLE crawler");// empty table for links
		$result->execute();
		$result=$pdo->prepare("TRUNCATE TABLE crawler_images"); // empty table for image links
		$result->execute();
		// insert sitename as first page to be parsed
		$result=$pdo->prepare("insert into crawler (url)values('$site')");
		$result->execute();
		$pdo= null;
                }
                catch (Exception $e) {   echo 'Caught exception: ',  $e->getMessage(), "\n";}
        }
	
	function parsepage(){
		$this->hardflush();	
		$html = new simple_html_dom();// new html parser
		$pdo = new DB_Pdo('vandersanden'); // new pdo object
		
		// fetch first not parsed page
		$result = $pdo->prepare('select id,url from crawler where parsed = 0 Limit 1');
		$result->execute();
		$pagesarray = $result->fetch(PDO::FETCH_ASSOC);
		$result = null;
		$pdo = null;
		
		// if no pages are found put $morepagestoparse at 1 so we know we can stop parsing
		if (count($pagesarray)==1){$this->morepagestoparse=1;}
		else{
			$id= $pagesarray ['id'];
			$url = $pagesarray['url'];
			if($this->url_exists($url)){// if page is found, parse it
				if ($this->is_php($url)){$this->displayicon("php",$url);}// geef icoon voor php pagina
				if ($this->is_html($url)){$this->displayicon("html",$url);} // geef icoon voor html pagina
			
				// find all links on the page
				$html->load_file($url);
				$links = $html->find('a[href]');
				$linklist = array();
				foreach ($links as $link){array_push($linklist,$link->href);}
				 $linklist = $this->cleanuplinks($linklist,$url); // cleanup the links
				 $linklist = $this->return_only_pages($linklist); //only use pages
				$linklist = $this->remove_urls_not_in_domain($linklist,$this->site); // only use urls from this domain
				$linklist = array_diff($linklist,$this->urlsfromlastpage);
				$this->urlsfromlastpage = array_merge($linklist,$this->urlsfromlastpage);
				$this->urlsfromlastpage = array_unique($this->urlsfromlastpage);
				foreach($linklist as $link){
					$pdo = new DB_Pdo('vandersanden');
					try{ // check if the link is already in the database, if yes then $
						$pdo->quote($link);
						if($result= $pdo->query("SELECT COUNT(url) FROM crawler WHERE url = '$link'")){
							if($result->fetchColumn()>0){ $n = 1;}else{$n = 0; }
						}
					}
					catch (PDOStatement $e){echo $e->getMessage();}
				
					if($n>0 or $protocol=="mailto:"){}
					else{
					/*if(!$protocol=="mailto:"){*/
						$filename = $this->determine_name($link);
						$result=$pdo->exec("insert ignore into crawler (url,links,name) values('$link','$id','$filename')");
						$this->request++;
						time_nanosleep(0, 300000000);// sleep 1/10 a second
					}
				}
				$result = null;
				$pdo = null;
					
				// find all images on page
				$images = $html->find('img');
				$imagelist = array();
				foreach ($images as $image){array_push($imagelist, $image->src);}
				$imagelist = $this->cleanuplinks($imagelist,$url); // cleanup the links
				$imagelist = $this->remove_urls_not_in_domain($imagelist,$this->site); // only use urls from this domain
				
				foreach($imagelist as $src){
					$pdo = new DB_Pdo('vandersanden');
					// check that the src starts with http://
					$protocol = substr($src,0,7);
					try{ // check if the link is already in the database, if yes then $
						$pdo->quote($src);
						$result= $pdo->query("SELECT COUNT(url) FROM crawler_images WHERE url = '$src'");
						if($result->fetchColumn()>0){ $n = "found";}else{$n = "notfound"; }
						
					}
					catch (PDOStatement $e){echo $e->getMessage();}
				
   					if($n=="notfound"){
						//$filename = $this->determine_name($src);
						if($this->url_exists($src)){
							$exists = 1; 
							$this->displayicon("image",$src);
						}
						else{
							$exists = 2; $this->displayicon("imageerror",$src);
						}
						$result=$pdo->exec("insert into crawler_images (url,crawlid,exist) values('$src','$id','$exists')");
						time_nanosleep(0, 300000000);// sleep 1/10 a second
						unset($n);
					}
					$result = null;
					$pdo = null;
					
				}
				// pagina is done, put parsed on 1 so we don't parse this page again
				$pdo = new DB_Pdo('vandersanden');
				$pdo->exec("update crawler set parsed=1 where id ='$id'");
				$pdo = null;
			

			}
			else{
				// page not found put parsed on 2 so we know this page is a 404
				$pdo = new DB_Pdo('vandersanden');
				$pdo->exec("update crawler set parsed=2 where id ='$id'");
				$pdo = null;
				$this->displayicon("pageerror",$url);
			}
		
		$this->hardflush();	
		time_nanosleep(0, 300000000);// sleep 1/10 a second
		}
		$html->clear();
		unset($query);
		unset($id);
		unset($links);
		unset($mylink);
		unset($myarray);
		unset($html);
		unset($pdo);
		unset($result);
		
	}
	
	
	function url_exists($url){
		/* checks if a page exists*/
		$handle = @fopen($url,'r');
		if($handle !== false){ return true;}  
		else {return false;}
	}



	function displayicon($type,$url){
		switch ($type){
			case "php":
				//echo "<a href='$url' title='$url'><img src='http://www.designexpress.be/images/famfamfam/page_white_php.png' style='border: 0px;'/></a>";
				break;
			case "html":
				//echo "<a href='$url' title='$url'><img src='http://www.designexpress.be/images/famfamfam/page_white.png' style='border: 0px;'/></a>";
				break;
			case "asp":
				//echo "<a href='$url' title='$url'><img src='http://www.designexpress.be/images/famfamfam/page_white_asp.png' style='border: 0px;'/></a>";
				break;
			case "pageerror":	
				//echo "<a href=''><img src='http://www.designexpress.be/images/famfamfam/page_white_error.png' alt='$url'/></a>";
				break;
			case "image":
				//echo"<a href='$url' title='$url'><img src='http://www.designexpress.be/images/famfamfam/image.png' style='border: 0px;'/></a>";
				break;
			case "imageerror":
				//echo"<a href='$url' title='$url'><img src='http://www.designexpress.be/images/famfamfam/custom/image_error.png' style='border: 0px;'/></a>";
				break;
		}
			
			
	}
	function determine_name($link){
		/* rename function to prevent duplicate entrys in the database*/
		$basename = basename($link);
		//echo "basename: ".$basename;
		if ($basename=="index.php" or $basename =="index.asp" or $basename =="index.html" or $basename=="index.htm"){
			$dirarray = explode("/",$link);
			$lastitem = count($dirarray)-2;
			$returnname = $dirarray[$lastitem];
		}
		else{$returnname = $basename;}
		//echo " returnname: ".$returnname."\n";
		return $returnname;
	}


	function relative_to_abolute_url($absolute, $relative) {
		/* converts a relative path to an absolute */
        $p = parse_url($relative);
        if($p["scheme"])return $relative;
        extract(parse_url($absolute));
        $path = dirname($path); 
        if($relative{0} == '/'){
            //$cparts = array_filter(explode("/", $relative));
            return $absolute.$relative;
            }
        else {
            $aparts = array_filter(explode("/", $path));
            $rparts = array_filter(explode("/", $relative));
            $cparts = array_merge($aparts, $rparts);
            foreach($cparts as $i => $part) {
                if($part == '.'){$cparts[$i] = null;}
                if($part == '..'){$cparts[$i - 1] = null; $cparts[$i] = null;}
            }
            $cparts = array_filter($cparts);
        }
        $path = implode("/", $cparts);
        $url = "";
        if($scheme){$url = "$scheme://";}
        if($user){
            $url .= "$user";
            if($pass){$url .= ":$pass";}
            $url .= "@";
        }
        if($host){$url .= "$host/";}
        $url .= $path;
        return $url;
	}
	
	
	function is_mailto($link){
		$protocol = substr($link,0,7);
		if ($protocol == "mailto:"){ return true;}
		else{return false;}
	}

	function is_http($link){
		$protocol = substr($link,0,7);
		if ($protocol == "http://"){ return true;}
		else{return false;}
	}

	function is_https($link){
		$protocol = substr($link,0,8);
		if ($protocol == "https://"){ return true;}
		else{return false;}
	}
	
	function is_anchor($link){
		$protocol = substr($link,0,1);
		if ($protocol == "#"){ return true;}
		else{return false;}	
	}
	
	function is_javascript($link){
		$protocol = substr($link,0,10);
		if ($protocol == "javascript:"){ return true;}
		else{return false;}	
	}

	function is_page($page){
		$page = strtolower($page);
		$extension = substr($page,-5);
		$findhtm = stripos($extension,"htm");
		$findphp = stripos($extension,"php");
		$findasp = stripos($extension,"asp");
		$checkforslash = substr($page,-1);
		if($checkforslash == "/"){ 
			
			$findfolder = true;
		}
		else{
			
			$findfolder = false;
		}
		if($findhtm === false and $findphp === false and $findasp === false and $findfolder === false){return false;}
		else{return true;}
	}
	
	function is_php($page){
		$extentsion = substr($page,-3);
		if ($extension = 'php'){return true;}else{return false;}
	}
	
	function is_asp($page){
		$extentsion = substr($page,-3);
		if ($extension = 'asp'){return true;}else{return false;}
	}
	
	function is_html($page){
		$extension = substr($page,-5);
		if (stripos($extension,"htm")){return true;}else{return false;}
	}
	
	
	function is_image($link){
		$link = strtolower($link);
		$extension = substr($link,-5);
		$findgif = strpos($extentsion,"gif");
		$findjpg = strpos($extentsion,"jpg");
		$findjpeg = strpos($extentsion,"jpeg");
		$findpng = strpos($extentsion,"png");
		unset($link);
		if($findgif=== false and $findjpg===false and $findjpeg===false and $findpng===false){ return false;}
		else{return true;}
	}
	
	function cleanuplinks($links,$parent){
		$cleanedlinks = array();
		foreach($links as $link){
			
			//if($str[strlen($link)-1] == "/"){$link = $link . "index.php";}
			if($this->is_http($link) || $this->is_https($link)){array_push($cleanedlinks,$link);}
			else{
				//if(!$this->is_javascript($link) and !$this->is_anchor($link) and ! $this->is_mailto($link)){
				if ($this->is_page($link)){
					//echo $link," - ",$parent," - ";
				$absolutelink = $this->relative_to_abolute_url($parent, $link);
				//echo $absolutelink,"\n";
				array_push($cleanedlinks,$absolutelink);
				}	
			}
		}
		return $cleanedlinks;
	}
	
	function return_only_pages($links){
		$onlypages = array();
		foreach ($links as $link){
			if($this->is_page($link)){array_push($onlypages,$link);}	
		}
		return $onlypages;
	}
	
	function remove_urls_not_in_domain($urls,$domain){
		$cleanedurls = array();
		foreach ($urls as $url){
			$length_domain = strlen($domain);
			$possibledomain= substr($url,0,$length_domain);
			if($possibledomain == $domain){array_push($cleanedurls,$url);}
		}	
		return $cleanedurls;
	}
	
	function parsesite(){
		flush();
		while ($this->morepagestoparse<1){
			$this->parsepage();
			
			unset($pdo);
			unset($result);
			
		}
		$this->hardflush();	
		echo "# requests = $request ";
	}
	
	function hardflush($Out = "", $Padding = 4096){
		echo str_pad($Out, $Padding);
		@ob_flush();
		@flush();	
	}
}


?>